# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
from pymongo import MongoClient

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告


"""
途牛-海外旅游(内容、评论)
https://www.tuniu.com/abroad/

"""


class Test_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "User-Agent": ua.random
            # "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36",
        }


    def get_comments(self, pageNum):
        url = f"https://s.tuniu.com/search_complex/whole-as-0-%E9%A9%AC%E6%9D%A5%E8%A5%BF%E4%BA%9A/{pageNum}"
        html = HttpUtils.do_request("GET", url, self.headers, "")
        selector = etree.HTML(html.text)
        root = selector.xpath('//li/div[@class="theinfo"]')
        try:
            for r in root:
                dict_data = dict()
                url = r.xpath('./a/@href')[0]  # url
                id = url.split("/")[-1]
                dict_data['title'] = r.xpath('./a/dl/dt/p[@class="title"]/span/@name')[0]  # 标题
                dict_data['price'] = r.xpath('./a/div[@class="priceinfo"]/div[2]/div/em/text()')[0]  # 价格
                dict_data['manyi'] = "".join(r.xpath('./a/div[@class="priceinfo"]/div[2]/div[2]/div/span//text()'))  # 满意度
                dict_data['chuyou'] = "".join(r.xpath('./a/div[@class="priceinfo"]/div[2]/div[2]/div[2]/p[1]//text()'))  # 出游人数
                dict_data['dianping'] = "".join(r.xpath('./a/div[@class="priceinfo"]/div[2]/div[2]/div[2]/p[2]//text()'))  # 点评人数
                print(str(dict_data))
                manyi = dict_data['manyi'].replace("%", "")
                if manyi != "":
                    if int(manyi) < 100:
                        self.get_detail(id, 1, 21, dict_data)
                        self.get_detail(id, 1, 22, dict_data)
        except:
            pass


    # 评论内容
    def get_detail(self, id, pagenum, typeid, parm_dict):
        postData = {
            "page": pagenum,  # "2",
            "productId": id,  #"210606443",
            "selectedType": typeid,  #"0",
            "stamp": "053497922907215671579500683729"
        }
        url = "https://www.tuniu.com/papi/tour/comment/product"
        time.sleep(1)
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        count = data_json['data']['count']  # 总评论数量
        pageNum = data_json['data']['pageNum']  # 总页数
        if int(count) > 0:
            # 总评分
            # num = 0
            # for d1 in data_json['data']['selectedTypeList']:
            #     dict_data1 = {}
            #     num += 1
            #     dict_data1['tag_name_' + str(num)] = str(d1['selectedName'])
            #     dict_data1['tag_num_' + str(num)] = str(d1['selectedAmount'])
                # print(str(dict_data1))
            # 评论内容
            for d in data_json['data']['list']:
                dict_data = dict()
                dict_data['travelTypeName'] = str(d['travelTypeName'])  # 旅行类型
                dict_data['realName'] = str(d['realName'])  # 用户昵称
                dict_data['remarkTime'] = str(d['remarkTime']) # 评论时间
                dict_data['content'] = str(d['content'])  # 评论内容
                dict_data['compName'] = str(d['compName'])  # 总体评价
                num2 = 0
                for d2 in d['subGradeContents']:
                    dict_data['notes_' + str(num2)] = str(d2['notes'])
                    dict_data['dataIvalue_' + str(num2)] = str(d2['dataIvalue'])
                    num2 += 1
                url_child = "https://www.tuniu.com/tour/"+ str(id)
                dict_data['url_child'] = url_child
                dict_data['title'] = parm_dict['title']
                dict_data['manyi'] = parm_dict['manyi']
                dict_data['price'] = parm_dict['price']
                dict_data['country'] = "马来西亚"
                print(str(dict_data))
                self.insertItem("tuniu", dict_data)
            # 翻页
            if pagenum < pageNum:
                pagenum += 1
                self.get_detail(id, pagenum, typeid, parm_dict)

    def insertItem(self, tableName, data):
        my_set = db[tableName]
        my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["Test"]
    test_get = Test_Get()
    for i in range(1, 20):
        print(f"========第{i}页===========")
        test_get.get_comments(i)
    # test_get.get_detail("210606443", "1")